#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/types.h>
#include <linux/errno.h>
#include <linux/fcntl.h>
#include <linux/mm.h>
#include <linux/mman.h>
#include <linux/miscdevice.h>
#include <linux/proc_fs.h>
#include <linux/device.h>
#include <linux/fs.h>
#include <linux/slab.h>
#include <asm/io.h>
#include <linux/spinlock.h>
#include <linux/vmalloc.h>
#include <asm/cacheflush.h>
#include <linux/seq_file.h>
#include <asm/io.h>
#include <linux/string.h>
#include <linux/list.h>
#include <linux/dma-mapping.h>
#include <linux/highmem.h>
#include <linux/dma-contiguous.h>
#include <asm/memory.h>
#include <asm/tlbflush.h>
#include <asm/pgtable.h>
#include <linux/dma-buf.h>
#include <linux/file.h>
#include <linux/miscdevice.h>
#include <linux/dma-direction.h>
#include <linux/platform_device.h>
#include <linux/of_platform.h>
#include <linux/syscalls.h>
#include "uapi/linux/ion.h"

#include "ei_os.h"
#include "os_memory.h"

#define log_info(s...) do{ os_printk(s); }while(0)
#define log_err(s...) do{ os_printk(s); }while(0)
#define log_warn(s...) do{ os_printk(s); }while(0)

#if defined (ARCH_LOMBO_N7V3)
#define FLUSH_CACHE_TH (256*1024)
#elif defined (ARCH_LOMBO_N7V7)
#define FLUSH_CACHE_TH (128*1024)
#else
#define FLUSH_CACHE_TH (128*1024)
#endif

OSAL_LIST_HEAD(mem_list);
static struct platform_device *mem_pdev;

static DEFINE_SEMAPHORE(mem_lock);

static struct mem_stats_info mem_stats = {0};

struct os_mem_user_info {
	struct os_list_head list;
	struct semaphore sem;
	pid_t pid;
	pid_t map_pid;
};

static void __os_mem_block_free(os_mb_t *mb);

static int __get_heap_id(unsigned int type)
{
	int ret = -1, i;
	struct ion_heap_query query;
	struct ion_heap_data data[ION_HEAP_TYPE_CUSTOM];

	memset(&query, 0, sizeof(query));
	query.cnt = ION_HEAP_TYPE_CUSTOM;
	query.heaps = (__u64)(unsigned long)&data[0];

	ret = ion_query_heaps_kernel(&query);
	if (ret < 0) {
		log_err("query err %d\n", ret);
		return ret;
	}
#if 0
	for (i = 0; i < query.cnt; i++)
		log_err("heap name %s, type %d, id %d\n",
			data[i].name, data[i].type, data[i].heap_id);

#endif
	for (i = 0; i < query.cnt; i++) {
		if (data[i].type == type) {
			return data[i].heap_id;
		}
	}

	return ret;
}

static struct mem_block_info *__addr2mbi(unsigned long addr,
	struct os_mem_user_info *mui)
{
	struct mem_block_info *p = NULL;
	struct mem_block_info *ret = NULL;

	os_list_for_each_entry(p, &mui->list, list) {
		if ((addr >= p->phys_addr) && addr < (p->phys_addr + p->size)) {
			ret = p;
			break;
		}
	}

	return ret;
}

static struct mem_block_info *__virt2mbi(void *addr,
	struct os_mem_user_info *mui)
{
	struct mem_block_info *mbi = NULL;
	struct mem_block_info *ret = NULL;

	os_list_for_each_entry(mbi, &mui->list, list) {
		if (((unsigned long)addr >= (unsigned long)mbi->uvirt_addr)
			&& ((unsigned long)addr < (unsigned long)(mbi->uvirt_addr + mbi->size))) {
			ret = mbi;
			break;
		}
	}

	return ret;
}

static os_mb_t *__addr2mb(unsigned long addr)
{
	os_mb_t *p = NULL;
	os_mb_t *ret = NULL;

	down(&mem_lock);
	os_list_for_each_entry(p, &mem_list, list) {
		if ((addr >= p->phys_addr) && addr < (p->phys_addr + p->nbytes)) {
			ret = p;
			break;
		}
	}
	up(&mem_lock);

	return ret;
}

#if 0
static unsigned long __virt2phys(unsigned long addr,
	struct os_mem_user_info *mui)
{
	struct mem_block_info *mbi = NULL;
	unsigned long ret = 0;

	os_list_for_each_entry(mbi, &mui->list, list) {
		if (((unsigned long)addr >= (unsigned long)mbi->uvirt_addr)
			&& ((unsigned long)addr < (unsigned long)(mbi->uvirt_addr + p->size))) {
			ret = mbi->phys_addr;
			break;
		}
	}

	return ret;
}
#endif

static int __add_list_sort_by_addr(os_mb_t *mb)
{
	os_mb_t *p = NULL;

	os_list_for_each_entry(p, &mem_list, list) {
		if (mb->phys_addr < p->phys_addr)
			break;
	}

	os_list_add(&mb->list, p->list.prev);

	return 0;
}

static struct mem_block_info *__os_mem_alloc_mbi(os_mb_t *mb, struct mem_block_info *umbi,
	struct os_mem_user_info *mui)
{
	struct mem_block_info *mbi_new = NULL;

	if (IS_ERR_OR_NULL(mb)) {
		return ERR_PTR(-EINVAL);
	}

	mbi_new = os_kmalloc(sizeof(*mbi_new), os_gfp_kernel);
	if (mbi_new == NULL) {
		return ERR_PTR(-ENOMEM);
	}

	memcpy(mbi_new, umbi, sizeof(*mbi_new));
	mbi_new->phys_addr = mb->phys_addr;
	mbi_new->size = mb->nbytes;

	mbi_new->mb = (os_hdl)mb;
	mbi_new->prot = PROT_READ;
	mbi_new->flags = MAP_SHARED;
	os_list_add_tail(&mbi_new->list, &mui->list);

	return mbi_new;
}


static void *__os_mem_map_kernel(phys_addr_t phys_addr, int nbytes, int cache)
{
	int j;
	void *vaddr;
	pgprot_t pgprot;
	int npages = PAGE_ALIGN(nbytes) / PAGE_SIZE;
	struct page **pages = os_vmalloc(array_size(npages,
				sizeof(struct page *)));
	struct page **tmp = pages;
	struct page *page = phys_to_page(phys_addr);

	if (!pfn_valid(__phys_to_pfn(phys_addr))) {
		log_err("invalid phys_addr 0x%p\n", (void *)phys_addr);
		return ERR_PTR(-EINVAL);
	}

	if (!pages) {
		return ERR_PTR(-ENOMEM);
	}

	if (cache) {
		pgprot = PAGE_KERNEL;
	} else {
		pgprot = pgprot_writecombine(PAGE_KERNEL);
	}

	for (j = 0; j < npages; j++) {
		*(tmp++) = page++;
	}

	vaddr = vmap(pages, npages, VM_MAP, pgprot);
	os_vfree(pages);

	if (!vaddr) {
		return ERR_PTR(-ENOMEM);
	}

	return vaddr;
}

static void __os_mem_unmap_kernel(void *virt)
{
	unsigned long vaddr = (unsigned long)virt;

	if ((vaddr >= VMALLOC_START) && (vaddr < VMALLOC_END)) {
		vunmap(virt);
	}
}

int __block_get(os_mb_t *mb)
{
	int ref;

	if (IS_ERR_OR_NULL(mb)) {
		return -1;
	}

	down(&mem_lock);

	ref = ++mb->ref;

	up(&mem_lock);

	return ref;
}

int __block_put(os_mb_t *mb)
{
	int ref = 0;

	if (IS_ERR_OR_NULL(mb)) {
		return -1;
	}

	down(&mem_lock);

	if (mb->ref > 0) {
		ref = --mb->ref;
		if (mb->ref == 0) {
			os_list_del(&mb->list);
			mb->cached |= EI_MB_MAP2KERN;
			mb->cached &= ~EI_MB_MAP2KERN_CACHED;
			__os_mem_block_free(mb);
		}
	} else {
		log_err("this block had beed release already!\n");
	}

	up(&mem_lock);

	return ref;
}

int __mbi_get(struct mem_block_info *mbi)
{
	int ref;

	if (IS_ERR_OR_NULL(mbi)) {
		return -1;
	}

	__block_get((os_mb_t *)mbi->mb);
	ref = ++mbi->ref;

	return ref;
}

int __mbi_put(struct mem_block_info *mbi)
{
	int ref = 0;

	if (IS_ERR_OR_NULL(mbi)) {
		return -1;
	}

	if (mbi->ref > 0) {
		__block_put((os_mb_t *)mbi->mb);
		ref = --mbi->ref;
		if (mbi->ref == 0) {
			os_list_del(&mbi->list);
			os_kfree(mbi);
		}
	} else {
		log_err("this block had beed released already!\n");
	}

	return ref;
}

static os_hdl __os_mem_block_alloc(const char *name,
	unsigned long size,
	unsigned long align,
	unsigned long gfp)
{
	os_mb_t *mb = NULL;
	unsigned int heap_id_mask = 0;
	unsigned int flags;
	unsigned int heap_id;
	struct dma_buf *dmabuf;
	struct dma_buf_attachment *attachment;
	struct sg_table *sg_table;
	dma_addr_t phys_addr;

	size = PAGE_ALIGN(size);
	heap_id = __get_heap_id(ION_HEAP_TYPE_DMA);
	heap_id_mask |= (1 << heap_id);
	heap_id = __get_heap_id(ION_HEAP_TYPE_CARVEOUT);
	heap_id_mask |= (1 << heap_id);
	flags = 0;
	dmabuf = ion_alloc_kernel(size, heap_id_mask, flags);
	if (IS_ERR(dmabuf)) {
		log_err("ion_alloc fail! size=0x%lx, heap_id_mask=0x%x\n",
			size, heap_id_mask);
		goto exit;
	}

	attachment = dma_buf_attach(dmabuf, &mem_pdev->dev);
	if (!attachment) {
		log_err("failed attach dmabuf.\n");
		dma_buf_put(dmabuf);
		return -EFAULT;
	}
	sg_table = dma_buf_map_attachment(attachment,
			DMA_BIDIRECTIONAL);
	if (!sg_table) {
		log_err("failed map dmabuf attachment.\n");
		dma_buf_detach(dmabuf, attachment);
		dma_buf_put(dmabuf);
		return -EFAULT;
	}
	phys_addr = sg_phys(sg_table->sgl);
	dma_buf_unmap_attachment(attachment,
		sg_table, DMA_BIDIRECTIONAL);
	dma_buf_detach(dmabuf, attachment);

	mb = os_kzalloc(sizeof(os_mb_t), os_gfp_kernel);
	if (mb == NULL) {
		log_err("alloc memory fail.\n");
		goto free;
	}
	memset(mb, 0, sizeof(os_mb_t));

	if (name) {
		strlcpy(mb->name, name, EI_MB_NAME_LEN);
	} else {
		strncpy(mb->name, "<null>", EI_MB_NAME_LEN - 1);
	}
	OSAL_INIT_LIST_HEAD(&mb->list);
	mb->phys_addr = phys_addr;
	mb->nbytes = size;
	mb->dmabuf = dmabuf;

exit:
	return (os_hdl)mb;

free:
	dma_buf_put(dmabuf);
	return (os_hdl)NULL;
}

static void __os_mem_block_free(os_mb_t *mb)
{
	if (IS_ERR_OR_NULL(mb)) {
		log_err("invalid hdl=0x%p\n", mb);
	} else {
		dma_buf_put(mb->dmabuf);
		os_kfree(mb);
	}
}

static int __os_mem_flush_dcache(void *virt, int nbytes)
{
	unsigned long virt_tmp = (unsigned long)virt;

	if (IS_ERR_OR_NULL(virt)) {
		return -EINVAL;
	}

	virt_tmp = ALIGN_DOWN(virt_tmp, CACHE_LINE_SIZE);
	nbytes = ALIGN(nbytes + (virt_tmp - (unsigned long)virt), CACHE_LINE_SIZE);

    if (nbytes > FLUSH_CACHE_TH) {
#if defined(CONFIG_64BIT)
        //flush_cache_all();
#else
        __cpuc_flush_kern_all();
#endif
    } else {
#if defined(CONFIG_64BIT)
        __flush_dcache_area((void *)virt_tmp, nbytes);
#else
        __cpuc_flush_dcache_area((void *)virt_tmp, nbytes);
#endif
    }

	return 0;
}

os_hdl os_mem_block_alloc(const char *name,
	unsigned long size,
	unsigned long align,
	unsigned long gfp)
{

	os_mb_t *mb = NULL;

	mb = (os_mb_t *)__os_mem_block_alloc(name, size, align, gfp);
	if (IS_ERR_OR_NULL(mb)) {
		log_err("mem block alloc fail, ret=%ld\n", (unsigned long)mb);
		goto exit;
	}

	__block_get(mb);

	down(&mem_lock);
	__add_list_sort_by_addr(mb);
	up(&mem_lock);

exit:
	return (os_hdl)mb;
}
EXPORT_SYMBOL(os_mem_block_alloc);

void os_mem_block_free(os_hdl hdl)
{
	os_mb_t *mb = (os_mb_t *)hdl;

	if (IS_ERR_OR_NULL(mb)) {
		log_err("invalid hdl=0x%p\n", mb);
	}  else {
		__block_put(mb);
	}
}
EXPORT_SYMBOL(os_mem_block_free);

void os_mem_block_free_by_phys(unsigned long phys_addr)
{
	os_mb_t *mb = NULL;

	mb = __addr2mb(phys_addr);
	if (IS_ERR_OR_NULL(mb)) {
		log_err("invalid hdl=0x%p\n", mb);
	} else {
		__block_put(mb);
	}
}
EXPORT_SYMBOL(os_mem_block_free_by_phys);

unsigned long os_mem_block_phys(os_hdl hdl)
{
	os_mb_t *mb = (os_mb_t *)hdl;

	if (IS_ERR_OR_NULL(mb)) {
		log_err("invalid hdl=0x%p\n", mb);
		return 0;
	}

	return mb->phys_addr;
}
EXPORT_SYMBOL(os_mem_block_phys);

void *os_mem_block_map_kernel_cache(os_hdl hdl)
{
	os_mb_t *mb = (os_mb_t *)hdl;
	void *virt_addr = NULL;

	if (IS_ERR_OR_NULL(mb)) {
		log_err("invalid hdl=0x%p\n", mb);
		return NULL;
	}

	down(&mem_lock);
	virt_addr = __os_mem_map_kernel(mb->phys_addr, mb->nbytes, 1);
	mb->kvirt_addr = virt_addr;
	mb->cached |= EI_MB_MAP2KERN;
	mb->cached |= EI_MB_MAP2KERN_CACHED;
	up(&mem_lock);

	return virt_addr;
}
EXPORT_SYMBOL(os_mem_block_map_kernel_cache);

void *os_mem_block_map_kernel_noncache(os_hdl hdl)
{
	os_mb_t *mb = (os_mb_t *)hdl;
	void *virt_addr = NULL;

	if (IS_ERR_OR_NULL(mb)) {
		log_err("invalid hdl=0x%p\n", mb);
		return NULL;
	}

	down(&mem_lock);
	virt_addr = __os_mem_map_kernel(mb->phys_addr, mb->nbytes, 0);
	mb->kvirt_addr = virt_addr;
	mb->cached |= EI_MB_MAP2KERN;
	mb->cached &= ~EI_MB_MAP2KERN_CACHED;
	up(&mem_lock);

	return virt_addr;
}
EXPORT_SYMBOL(os_mem_block_map_kernel_noncache);

void *os_mem_map_kernel_cache(unsigned long phys_addr, int nbytes)
{
	void *virt_addr = NULL;

	down(&mem_lock);
	virt_addr = __os_mem_map_kernel(phys_addr, nbytes, 1);
	up(&mem_lock);

	return virt_addr;
}
EXPORT_SYMBOL(os_mem_map_kernel_cache);

void *os_mem_map_kernel_noncache(unsigned long phys_addr, int nbytes)
{
	void *virt_addr = NULL;

	down(&mem_lock);
	virt_addr = __os_mem_map_kernel(phys_addr, nbytes, 0);
	up(&mem_lock);

	return virt_addr;
}
EXPORT_SYMBOL(os_mem_map_kernel_noncache);

void os_mem_block_unmap_kernel(os_hdl hdl)
{
	os_mb_t *mb = (os_mb_t *)hdl;

	if (IS_ERR_OR_NULL(mb)) {
		log_err("invalid hdl=0x%p\n", mb);
		return ;
	}

	down(&mem_lock);
	__os_mem_unmap_kernel(mb->kvirt_addr);
	mb->kvirt_addr = 0;
	mb->cached &= ~EI_MB_MAP2KERN;
	mb->cached &= ~EI_MB_MAP2KERN_CACHED;

	up(&mem_lock);
}
EXPORT_SYMBOL(os_mem_block_unmap_kernel);

void os_mem_unmap_kernel(void *kvirt_addr)
{
	if (IS_ERR_OR_NULL(kvirt_addr)) {
		log_err("invalid kvrit=0x%p\n", kvirt_addr);
		return ;
	}

	down(&mem_lock);
	__os_mem_unmap_kernel(kvirt_addr);
	up(&mem_lock);
}
EXPORT_SYMBOL(os_mem_unmap_kernel);

int os_mem_flush_dcache(void *kvirt_addr, unsigned long phys_addr, int nbytes)
{
	int ret;

	ret = __os_mem_flush_dcache(kvirt_addr, nbytes);

	return ret;
}
EXPORT_SYMBOL(os_mem_flush_dcache);

int os_mem_invalid_dcache(void *kvirt_addr, unsigned long phys_addr, int nbytes)
{
	int ret;

	ret = __os_mem_flush_dcache(kvirt_addr, nbytes);

	return ret;
}
EXPORT_SYMBOL(os_mem_invalid_dcache);

void os_mem_get_usr_memstats(struct mem_stats_info *pmem_stats)
{
	*pmem_stats = mem_stats;
}
EXPORT_SYMBOL(os_mem_get_usr_memstats);

static int os_mem_mmap(struct file *file, struct vm_area_struct *vma)
{
	struct mem_block_info *mbi;
	struct os_mem_user_info *mui = file->private_data;
	unsigned long offset = vma->vm_pgoff << PAGE_SHIFT;
	int cache = 0;

	mbi = __addr2mbi(offset, mui);

	if (IS_ERR_OR_NULL(mbi)) {
		return -EINVAL;
	} else {
		if (mbi->uvirt_addr) {
			log_err("mb(0x%lx) has been mapped!\n", offset);
			return -EIO;
		}
		cache = mbi->map_cached;
	}

	if (file->f_flags & O_SYNC) {
		vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
	} else {
#if defined(CONFIG_64BIT)
		vma->vm_page_prot = __pgprot(pgprot_val(vma->vm_page_prot)
				| PTE_WRITE | PTE_DIRTY);
#else
		vma->vm_page_prot = __pgprot(pgprot_val(vma->vm_page_prot)
				| L_PTE_PRESENT | L_PTE_YOUNG
				| L_PTE_DIRTY | L_PTE_MT_DEV_CACHED);
#endif

		if (cache == 0) {
			vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
		}
	}

	if (remap_pfn_range(vma,
			vma->vm_start,
			vma->vm_pgoff,
			vma->vm_end - vma->vm_start,
			vma->vm_page_prot)) {
		return -EAGAIN;
	}

		return 0;
	}

#ifdef CONFIG_PROC_FS
#define OS_MEM_PROC_NAME "ei_mem"
static int os_mem_proc_show(struct seq_file *sfile, void *unused)
{
    os_mb_t *p = NULL;
    unsigned int mb_num = 0, i = 0, bits_size = 0;
    unsigned int used_size = 0;
    unsigned long addr_min = 0xffffffff, addr_max = 0;
    unsigned char *bitmap = NULL;

    down(&mem_lock);

#define MEM_PROC_FMT "name=\"%16s\", phys(0x%08lX, 0x%08lX), kvirt=0x%p(%12s, %12s), size=%5luKB, ref=%2u"
#define os_mem_arg(p) (p)->name, (p)->phys_addr, (p)->phys_addr+(p)->nbytes-1, (p)->kvirt_addr, \
	(EI_MB_MAP2KERN & (p)->cached) ? (((p)->cached & EI_MB_MAP2KERN_CACHED) ? "cache[k]" : "non-cache[k]") : "non-map[k]" , \
	(EI_MB_MAP2USR & (p)->cached) ? (((p)->cached & EI_MB_MAP2USR_CACHED) ? "cache[u]" : "non-cache[u]") : "non-map[u]", \
	(p)->nbytes/1024, (p)->ref

	list_for_each_entry(p,&mem_list, list) {
		seq_printf(sfile, "MB: " MEM_PROC_FMT "\n", os_mem_arg(p));
		used_size += p->nbytes / 1024;
		++mb_num;
        if (addr_min >= p->phys_addr) {
            addr_min =  p->phys_addr & 0xfff00000;
        }
        if (addr_max <= p->phys_addr + p->nbytes) {
            addr_max =  p->phys_addr + p->nbytes;
        }

	}

	seq_printf(sfile, "\nsummary:\n"
	    " memory used=%dKB(%dMB+%dKB), block_number=%d\n",
	    used_size, used_size / 1024, used_size % 1024, mb_num);

	bits_size =  (addr_max - addr_min) / PAGE_SIZE;
	bitmap = os_vmalloc(bits_size);
	if (bitmap) {
		os_memset(bitmap, 0, bits_size);
		list_for_each_entry(p,&mem_list, list) {
			for (i = 0; i < p->nbytes / PAGE_SIZE; ++i) {
				bitmap[i + (p->phys_addr - addr_min) / PAGE_SIZE] = 1;
			}
		}

		seq_printf(sfile, "\n-------------------------------------------------------"
			"----------MemBitMap Start--------------------------"
			"----------------------------------------------------\n");
		for (i = 0; i < bits_size; ++i) {
			if (i % (1024 * 1024 / PAGE_SIZE) == 0)
				seq_printf(sfile, "\nphys(0x%08lX) offs(%03ld.0MB)",
					addr_min + i * PAGE_SIZE, i * PAGE_SIZE / 1024 / 1024);
			else if(i % (1024 * 1024 / PAGE_SIZE / 2) == 0)
				seq_printf(sfile, "\nphys(0x%08lX) offs(%03ld.5MB)",
					addr_min + i * PAGE_SIZE, i * PAGE_SIZE / 1024 / 1024);

			if (bitmap[i] == 0) {
				seq_printf(sfile, " ");
			} else {
				seq_printf(sfile, ".");
			}
		}
		seq_printf(sfile, "\n\n--------------------------------------------------------"
			"---------MemBitMap End------------------------------"
			"--------------------------------------------------\n");
		os_vfree(bitmap);
	}
	up(&mem_lock);

	return 0;
}

static int os_mem_proc_open(struct inode *inode, struct file *file)
{
	return single_open(file, os_mem_proc_show, NULL);
}

static const struct file_operations os_mem_proc_ops = {
	.owner = THIS_MODULE,
	.open           = os_mem_proc_open,
	.read           = seq_read,
	.llseek         = seq_lseek,
	.release        = single_release,
};

static int __init os_mem_proc_init(void)
{
    struct proc_dir_entry *pde;

    pde = proc_create(OS_MEM_PROC_NAME, 0, NULL, &os_mem_proc_ops);
    if (!pde) {
        log_err("cannot create %s procfs entry!\n", OS_MEM_PROC_NAME);
        return -1;
    }
    return 0;
}

static void __exit os_mem_proc_exit(void)
{
    remove_proc_entry(OS_MEM_PROC_NAME, NULL);
}

#else
static int __init os_mem_proc_init(void){ return 0; }
static void __exit os_mem_proc_exit(void){ }

#endif /* CONFIG_PROC_FS */

static int ioctl_mem_alloc(struct file * file,
	struct mem_block_info * umbi)
{
	struct os_mem_user_info *mui = file->private_data;
	struct mem_block_info *mbi;
	os_mb_t *mb;

	mb = (os_mb_t *)__os_mem_block_alloc(umbi->name, umbi->size, umbi->align, umbi->gfp);
	if (IS_ERR_OR_NULL(mb)) {
		log_err("mem block alloc fail, ret=%ld\n", (unsigned long)mb);
		return -ENOMEM;
	}

	mbi = __os_mem_alloc_mbi(mb, umbi, mui);
	if (IS_ERR_OR_NULL(mbi)) {
		__os_mem_block_free(mb);
		log_err("alloc mbi failed!\n");
		return -ENOMEM;
	}

	__mbi_get(mbi);
	down(&mem_lock);
	__add_list_sort_by_addr(mb);
	mem_stats.alloc_cnt++;
	up(&mem_lock);

	umbi->phys_addr = mbi->phys_addr;

	return 0;
}

static int ioctl_mem_free(struct file * file,
	struct mem_block_info * umbi)
{
	struct os_mem_user_info *mui = file->private_data;
	struct mem_block_info *mbi;

	mbi = __addr2mbi(umbi->phys_addr, mui);
	if (IS_ERR_OR_NULL(mbi)) {
		log_err("invalid phys addr 0x%lx!\n", umbi->phys_addr);
		return -ENOMEM;
	}

	__mbi_put(mbi);

	down(&mem_lock);
	mem_stats.free_cnt++;
	up(&mem_lock);

	return 0;
}

static int ioctl_mem_remap(struct file * file,
	struct mem_block_info * umbi,
	int cached)
{
	struct os_mem_user_info *mui = file->private_data;
	struct mem_block_info *mbi;
	unsigned long addr, len, prot, flags, pgoff;
	os_mb_t *mb = NULL;

	if ((mbi = __addr2mbi(umbi->phys_addr, mui)) == NULL) {
		mb = __addr2mb(umbi->phys_addr);
		if (IS_ERR_OR_NULL(mb)) {
			log_err("cannot find mb, phys=0x%lx\n", umbi->phys_addr);
		}
		/* maybe the physical address alloc by kernel driver */
		mbi = __os_mem_alloc_mbi(mb, umbi, mui);
		if (IS_ERR_OR_NULL(mbi)) {
			log_err("cannot alloc mbi, phys=0x%lx\n", umbi->phys_addr);
			return PTR_ERR(mbi);
		}
	}

	mb = (os_mb_t *)mbi->mb;
	if (mbi->uvirt_addr && (mbi->map_ref > 0)) {
		if (cached != mbi->map_cached) {
			log_err("block<%s> already mapped %s, cannot remap %s.\n",
				mb->name,
				mbi->map_cached ? "cached" : "non-cached",
				cached ? "cached" : "non-cached");
			return -EINVAL;
		}

		__mbi_get(mbi);
		mbi->map_ref++;

		umbi->uvirt_addr = mbi->uvirt_addr + (umbi->phys_addr - mbi->phys_addr);
		return 0;
	}

	if (mbi->phys_addr & ~PAGE_MASK) {
		return -EINVAL;
	}

	len = PAGE_ALIGN(mbi->size);

	prot = umbi->prot;
	flags = umbi->flags;
	if (prot == 0) {
		prot = mbi->prot;
	}
	if (flags == 0) {
		flags = mbi->flags;
	}

	mui->map_pid = 0;
	mui->map_pid = current->pid;
	mbi->map_cached = cached;
	pgoff = mbi->phys_addr;
	addr  = vm_mmap(file, 0, len, prot, flags, pgoff);

	if (IS_ERR_VALUE(addr)) {
		log_err("vm_mmap(file, 0, %lu, 0x%lx, 0x%lx, 0x%lx) ret 0x%p\n",
			len, prot, flags, pgoff, (void *)addr);
		return addr;
	}

	mbi->uvirt_addr = (void *)addr;
	mbi->prot = prot;
	mbi->flags = flags;

	if (cached) {
		mb->cached |= EI_MB_MAP2USR;
		mb->cached |= EI_MB_MAP2USR_CACHED;
	} else {
		mb->cached |= EI_MB_MAP2USR;
		mb->cached &= ~EI_MB_MAP2USR_CACHED;
	}

	__mbi_get(mbi);
	mbi->map_ref++;

	down(&mem_lock);
	mem_stats.map_cnt++;
	up(&mem_lock);

	umbi->uvirt_addr = mbi->uvirt_addr + (umbi->phys_addr - mbi->phys_addr);

	return 0;
}

static int ioctl_mem_unmap(struct file * file, struct mem_block_info * umbi)
{
	int ret;
	unsigned long addr, len;
	struct mem_block_info *mbi;
	struct os_mem_user_info *mui = file->private_data;
	os_mb_t *mb = NULL;

	if ((mbi = __virt2mbi(umbi->uvirt_addr, mui)) == NULL) {
		return -EPERM;
	}

	if (!mbi->uvirt_addr) {
		log_warn("mb(0x%lx) isn't mapped into user-land!\n", mbi->phys_addr);
		umbi->uvirt_addr = NULL;
		return -EIO;
	}

	mb = (os_mb_t *)mbi->mb;

	mbi->map_ref--;

	if (mbi->map_ref > 0) {
		__mbi_put(mbi);
		return 0;
	}

	addr = (unsigned long)mbi->uvirt_addr;
	len  = PAGE_ALIGN(mbi->size);

	/* todo, before unmap,refresh cache manually */
	if (mbi->map_cached) {
		struct mm_struct *mm = current->mm;

		down_read(&mm->mmap_sem);
#if defined(CONFIG_64BIT)
		__flush_dcache_area((void *)addr, (size_t)len);
#else
		__cpuc_flush_dcache_area((void *)addr, (size_t)len);
#endif
		up_read(&mm->mmap_sem);
	}

	ret = vm_munmap(addr, len);
	mb->cached &= ~EI_MB_MAP2USR;
	mb->cached &= ~EI_MB_MAP2USR_CACHED;

	if (!IS_ERR_VALUE((unsigned long)ret)) {
		mbi->uvirt_addr = NULL;
		umbi->uvirt_addr = NULL;
	}

	__mbi_put(mbi);  /* If mbi will be release when mb_ref is ZERO */

	down(&mem_lock);
	mem_stats.unmap_cnt++;
	up(&mem_lock);

	return ret;
}

static int ioctl_mem_flush_dcache(struct file *file)
{
#if defined(CONFIG_64BIT)
	//on_each_cpu((smp_call_func_t)flush_cache_all, NULL, 1);
#else
	on_each_cpu((smp_call_func_t)__cpuc_flush_kern_all, NULL, 1);
#endif
	down(&mem_lock);
	mem_stats.flush_all_cnt++;
	up(&mem_lock);

    return 0;
}

static int ioctl_mem_flush_cache(struct file * file, struct mem_block_info * umbi)
{
	int ret = 0;
	unsigned long addr, len;
	struct mem_block_info *mbi;
	struct os_mem_user_info *mui = file->private_data;
	struct mm_struct *mm = current->mm;

	if ((mbi = __virt2mbi(umbi->uvirt_addr, mui)) == NULL) {
		return -EPERM;
	}

	if (!mbi->uvirt_addr) {
		log_warn("mb(0x%lx) isn't mapped into user-land!\n", mbi->phys_addr);
		umbi->uvirt_addr = NULL;
		return -EIO;
	}

	/* don't flush for noncached memory */
	if (mbi->map_cached == 0) {
		return 0;
	}

	addr = ALIGN_DOWN((unsigned long)umbi->uvirt_addr, CACHE_LINE_SIZE);
	len  = ALIGN(umbi->size + ((unsigned long)umbi->uvirt_addr - addr), CACHE_LINE_SIZE);
    if (len > FLUSH_CACHE_TH) {
        ret = ioctl_mem_flush_dcache(file);
    } else {
        down_read(&mm->mmap_sem);
#if defined(CONFIG_64BIT)
        __flush_dcache_area((void *)addr, (size_t)len);
#else
        __cpuc_flush_dcache_area((void *)addr, (size_t)len);
#endif
        up_read(&mm->mmap_sem);

        down(&mem_lock);
        mem_stats.flush_area_cnt++;
        up(&mem_lock);
    }

	return ret;
}

static int os_mem_open(struct inode * inode, struct file * file)
{
	struct os_mem_user_info *mui;

	mui = os_kmalloc(sizeof(*mui), os_gfp_kernel);
	if (mui == NULL) {
		log_err("alloc memory failed!\n");
		return -ENOMEM;
	}
	mui->pid = current->pid;
	mui->map_pid = 0;
	sema_init(&mui->sem, 1);
	OSAL_INIT_LIST_HEAD(&mui->list);

	file->private_data = (void *)mui;

	return 0;
}

static int os_mem_release(struct inode * inode, struct file * file)
{
	struct os_mem_user_info *mui = file->private_data;
	struct mem_block_info *mbi = NULL, *n = NULL;
	unsigned int i;

	down(&mui->sem);

	list_for_each_entry_safe(mbi, n, &mui->list, list) {
		for (i = 0; i < mbi->ref; i++) {
			__block_put((os_mb_t *)mbi->mb);
		}

		os_list_del(&mbi->list);
		os_kfree(mbi);
	}
	up(&mui->sem);

	file->private_data = NULL;
	os_kfree(mui);

	return 0;
}

static long os_mem_ioctl(struct file * file, unsigned int cmd, unsigned long arg)
{
	int ret = 0;
	struct os_mem_user_info *mui = file->private_data;
	struct mem_block_info mi = {0};

	down(&mui->sem);

	if ((_IOC_DIR(cmd) & _IOC_WRITE)) {
		if ((_IOC_SIZE(cmd) > sizeof(mi) || arg == 0)) {
			log_err("arg size=%d, arg=0x%lx\n", _IOC_SIZE(cmd), arg);
			ret = -EINVAL;

			goto exit;
		}
		if (copy_from_user(&mi, (void *)arg, _IOC_SIZE(cmd))) {
			log_err("copy_from_user fail.\n");
			ret = -EFAULT;

			goto exit;
		}
	}

	switch (_IOC_NR(cmd)) {
	case _IOC_NR(IOC_MB_ALLOC):
		ret = ioctl_mem_alloc(file, &mi);
		break;
		break;
	case _IOC_NR(IOC_MB_FREE):
		ret = ioctl_mem_free(file, &mi);
		break;

	case _IOC_NR(IOC_MB_USER_REMAP):
		ret = ioctl_mem_remap(file, &mi, 0);
		break;
	case _IOC_NR(IOC_MB_USER_REMAP_CACHED):
		ret = ioctl_mem_remap(file, &mi, 1);
		break;
	case _IOC_NR(IOC_MB_USER_UNMAP):
		ret = ioctl_mem_unmap(file, &mi);
		break;
	case _IOC_NR(IOC_MB_FLUSH_DCACHE_DIRTY):
		ret = ioctl_mem_flush_cache(file, &mi);
		break;
	case _IOC_NR(IOC_MB_FLUSH_DCACHE):
		ret = ioctl_mem_flush_dcache(file);
		break;
	default:
		log_err("invalid ioctl cmd = 0x%x\n", cmd);

		ret = -EINVAL;
		break;
	}

	if (!ret && (_IOC_DIR(cmd) & _IOC_READ)) {
		if (copy_to_user((void *)arg, &mi, _IOC_SIZE(cmd))) {
			printk("\ncopy_to_user fail.\n");
			ret = -EFAULT;

			goto exit;
		}
	}

exit:
	up(&mui->sem);
	return ret;
}

static struct file_operations os_mem_fops = {
	.owner	= THIS_MODULE,
	.open	 = os_mem_open,
	.release  = os_mem_release,
	.unlocked_ioctl = os_mem_ioctl,
	.mmap	 = os_mem_mmap,
};

static struct miscdevice os_mem_device = {
	.minor	= MISC_DYNAMIC_MINOR,
	.fops	= &os_mem_fops,
	.name	= "ei_mem"
};

static int os_mem_probe(struct platform_device * pdev)
{
	int ret;

	mem_pdev = pdev;
	ret = misc_register(&os_mem_device);
	if (ret) {
		log_err("register os_mem device fail!\n");
		return -1;
	}

	os_mem_proc_init();

	log_info("os_mem probe ok!\n");

	return 0;
}

int os_mem_remove(struct platform_device * pdev)
{
	os_mem_proc_exit();
	misc_deregister(&os_mem_device);

	return 0;
}

static const struct of_device_id os_mem_of_match[] = {
	{ .compatible = "ei, ei-mem", },
	{},
};

static struct platform_driver os_mem_driver = {
	.probe	  = os_mem_probe,
	.remove	 = os_mem_remove,
	.driver	 = {
		.name	   = "ei-mem",
		.of_match_table = os_mem_of_match,
		.owner	  = THIS_MODULE,
	},
};

int __init os_mem_init(void)
{
	return platform_driver_register(&os_mem_driver);
}

void __exit os_mem_exit(void)
{
	platform_driver_unregister(&os_mem_driver);
}
